# -*- coding: utf-8 -*-
import scrapy
import re
import json
from ..settings import GOODS, MAX_PAGE
from ..items import AllJdItem


class JdSpider(scrapy.Spider):
    page = MAX_PAGE
    name = 'jd'
    goods = GOODS
    # allowed_domains = ['www.xxx.com']
    # start_urls = []
    # base_url = f'https://search.jd.com/s_new.php?keyword={goods}&page=3&s=90&click=0'
    # 每一页商品列表url
    base_url = f'https://search.jd.com/s_new.php?keyword=%s&page=%d&s=%d&click=0'
    # 单个商品评论url(包括好评、差评、评论内容)
    comment_url = 'https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=%s&score=0&sortType=5&page=0&pageSize=10&isShadowSku=0&fold=1'

    def start_requests(self):
        for page in range(1, self.page):
            url = self.base_url % (self.goods, page, page * 30)
            yield scrapy.Request(url, dont_filter=True,)

    def parse(self, response):
        li_list = response.xpath('//*[@id="J_goodsList"]/ul/li')
        for li in li_list:
            item = AllJdItem()
            title = "_".join(li.xpath('./div/div[3]/a/em/text()').extract()).replace(" ","")
            price = li.xpath('./div/div[2]/strong/i/text()').extract_first()
            shop = li.xpath('./div/div[5]/span/a/text()').extract_first()
            produce_id = li.xpath('./@data-sku').extract_first()
            item['title'] = title
            item['price'] = price
            item['shop'] = shop

            yield scrapy.Request(url=self.comment_url % (produce_id), dont_filter=True, callback=self.comment_parse,
                                 meta={'item': item})

    def comment_parse(self, response):
        item = response.meta['item']
        page_data = response.text
        # 匹配评论数量信息部分
        comment = "{" + re.search(r'"productCommentSummary.*?},', page_data).group().rstrip(",") + "}"
        # 转成字典
        comment = json.loads(comment)
        # 总评论
        commentCount = comment['productCommentSummary']['commentCount']
        # 好评
        goodCount = comment['productCommentSummary']['goodCount']
        # 差评
        poorCount = comment['productCommentSummary']['poorCount']
        # 好评度
        goodRateShow = str(comment['productCommentSummary']['goodRateShow']) + '%'
        item['commentCount'] = commentCount
        item['goodCount'] = goodCount
        item['poorCount'] = poorCount
        item['goodRateShow'] = goodRateShow
        yield item
